# coding:utf-8
import math
import torch
import torch.nn as nn
from dataclasses import dataclass
from typing import Optional, Tuple
from enum import Enum
import torch.nn.functional as F

# VocabParallelEmbedding: This is mainly adapted from torch.nn.Embedding and all the default values are kept.
from fairscale.nn.model_parallel.layers import (
    ColumnParallelLinear,
    RowParallelLinear,
    VocabParallelEmbedding
)
import fairscale.nn.model_parallel.initialize as fs_init


class QuantizationScheme(Enum):
    int4_weight_int8_dynamic_activation = "int4_weight_int8_dynamic_activation"

@dataclass
class QuantizationArgs:
    scheme: Optional[QuantizationScheme] = None
    group_size: Optional[int] = None
    # ? 
    spinquant: bool = False

    # init attributes by args.
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            if k == 'scheme':
                setattr(self, k, QuantizationScheme(v))
            else:
                if hasattr(self, k):
                    setattr(self, k, v)

@dataclass
class LoRAArgs:
    rank: int
    scale: float
   
  
@dataclass
class ModelArgs:
    dim: int = 4096
    n_layers: int = 32
    n_heads: int = 32
    n_kv_heads: Optional[int] = None
    vocab_size: int = -1
    multiple_of: int = 256  # make SwiGLU hidden layer size multiple of large power of 2
    ffn_dim_multiplier: Optional[float] = None
    norm_eps: float = 1e-5
    rope_theta: float = 500000  # RoPE theta
    use_scaled_rope: bool = False # ?

    max_batch_size: int = 32
    max_seq_len: int = 2048  # 2K

    # vision model params
    vision_chunk_size: int = -1 # image resolution for image models
    vision_max_num_chunks: int = 4
    # ? used in _init_fusion_schedule (vision)
    vision_num_cross_attention_layers: int = -1

    quantization_args: Optional[QuantizationArgs] = None
    lora_args: Optional[LoRAArgs] = None

    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            if k == 'lora_args':
                setattr(self, k, LoRAArgs(**v))
            elif k == 'quantization_args':
                setattr(self, k, QuantizationArgs(**v))
            else:
                setattr(self, k, v)

        if self.n_kv_heads is None:
            self.n_kv_heads = self.n_heads

        # some configuration limitations.
        assert self.n_kv_heads <= self.n_heads
        assert self.n_heads % self.n_kv_heads == 0
        assert self.dim % self.n_heads == 0

def reshape_for_broadcast(freqs_cis:torch.Tensor, x:torch.Tensor):
    ndim = x.ndim
    assert 0 <= 1 < ndim
    assert freqs_cis.shape == (x.shape[1], x.shape[-1])
    # dim=1, dim=-1, set into 1. to broadcast at these dims.
    shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)]
    return freqs_cis.view(*shape)


def apply_rotary_emb(xq: torch.Tensor, xk:torch.Tensor,
                     freqs_cis: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
    # split latest dimension into (dim/2, 2)
    # as_complex: ai + b
    xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
    xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
    # xq_: bsz, seqlen, self.n_local_heads, self.head_dim //2, 2
    # --> bsz, 1, self.n_local_heads, self.head_dim //2, 1
    freqs_cis = reshape_for_broadcast(freqs_cis, xq_)
    # --> bsz, 1, self.n_local_heads, self.head_dim
    xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
    xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)

    return xq_out.type_as(xq), xk_out.type_as(xk)
    
def repeat_kv(x:torch.Tensor, n_rep: int):
    bs,sqlen,n_kv_heads, head_dim = x.shape
    if n_rep == 1:
        return x
    
    return (x[:,:,:,None,:]
            .expand(bs,sqlen,n_kv_heads, n_rep,head_dim)
            .reshape(bs,sqlen,n_kv_heads*n_rep,head_dim))


class Attention(nn.Module):
    def __init__(self, args: ModelArgs):
        super().__init__()
        self.n_kv_heads = args.n_kv_heads if args.n_kv_heads is not None else args.n_heads
        model_parallel_size = fs_init.get_model_parallel_world_size()
        # split heads into multi-gpus
        self.n_local_heads = args.n_heads // model_parallel_size
        self.n_local_kv_heads = self.n_kv_heads // model_parallel_size
        # GQA
        self.n_rep = self.n_local_heads // self.n_local_kv_heads
        self.head_dim = args.dim // args.n_heads
        
        # split dim of weight into multi-gpus. 
        # So we use the n_heads not local_heads
        self.wq = ColumnParallelLinear(
            args.dim,
            args.n_heads * self.head_dim,
            bias=False,
            gather_output=False,
            init_method=lambda x:x,
        )

        self.wk = ColumnParallelLinear(
            self.dim,
            self.n_kv_heads * self.head_dim,
            bias=False,
            gather_output=False,
            init_method=lambda x:x,
        )

        self.wv = ColumnParallelLinear(
            args.dim,
            self.n_kv_heads * self.head_dim,
            bias=False,
            gather_output=False,
            init_method=lambda x:x,
        )

        self.wo = RowParallelLinear(
            args.n_heads * self.head_dim,
            args.dim,
            bias=False,
            input_is_parallel=True,
            init_method=lambda x:x, 
        )

        self.cache_k = torch.zeros(
            (
                args.max_batch_size,
                args.max_seq_len,
                self.n_local_kv_heads,
                self.head_dim,
            )
        )

        self.cache_v = torch.zeros((
            args.max_batch_size,
            args.max_seq_len,
            self.n_local_kv_heads,
            self.head_dim,
        ))

    def forward(self,
                x: torch.Tensor,
                start_pos: int,
                freqs_cis: torch.Tensor,
                mask: Optional[torch.Tensor]):
        bsz, seqlen, _ = x.shape
        xq, xk, xv = self.wq(x),self.wk(x),self.wv(x)

        xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim)
        xk = xk.view(bsz, seqlen, self.n_local_heads, self.head_dim)
        xv = xv.view(bsz, seqlen, self.n_local_heads, self.head_dim)

        xq, xk = apply_rotary_emb(xq, xk, freqs_cis=freqs_cis)

        self.cache_k = self.cache_k.to(xq)
        self.cache_v = self.cache_v.to(xq)

        self.cache_k[:bsz, start_pos:start_pos + seqlen] = xk
        self.cache_v[:bsz, start_pos:start_pos + seqlen] = xv

        keys = self.cache_k[:bsz, :start_pos +seqlen]
        values = self.cache_v[:bsz, :start_pos + seqlen]
        # GQA, one to many.
        # repeat k/v heads if n_kv_heads < n_heads(Q)
        keys = repeat_kv(keys, self.n_rep)
        values = repeat_kv(values, self.n_rep)

        # -> bsz, n_local_head, seqlen, head_dim
        xq = xq.transpose(1, 2)
        # -> bsz, n_local_head, seqlen, head_dim
        keys = keys.transpose(1, 2)
        # -> bsz, n_local_head, seqlen, head_dim
        values = values.transpose(1, 2)
        scores = torch.matmul(xq, keys.transpose(2, 3)) / math.sqrt(self.head_dim)
        if mask is not None:
            # (bs, n_local_heads, seqlen, cache_len + seqlen)
            scores = scores + mask
        scores = F.softmax(scores.float(), dim=-1).type_as(xq)

        # (bs, n_local_heads, seqlen, head_dim)
        output = torch.matmul(scores, values)
        # bsz, seqlen, n_local_heads * head_dim
        output = output.transpose(1, 2).contiguous().view(bsz, seqlen, -1)

        return self.wo(output)

class FeedForward(nn.Module):
    def __init__(
        self,
        dim: int,
        hidden_dim: int,
        multiple_of: int,
        ffn_dim_multiplier: Optional[float],
    ):
        super().__init__()
        hidden_dim = int(2 * hidden_dim / 3)
        # custom dim factor multiplier
        if ffn_dim_multiplier is not None:
            hidden_dim = int(ffn_dim_multiplier * hidden_dim)
        hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)

        self.w1 = ColumnParallelLinear(dim, hidden_dim, bias=False, gather_output=False, init_method=lambda x:x)
        self.w2 = RowParallelLinear(hidden_dim, dim, bias=False, input_is_parallel=True, init_method=lambda x:x)
        self.w3 = ColumnParallelLinear(dim, hidden_dim, bias=False, gather_output=False, init_method=lambda x: x)

    def forward(self, x):
        # SwiGLU= (swish(xW) * xW) * W
        x = F.silu(self.w1(x))
        x = x * self.w3(x)
        return self.w2(x)



class TransformerBlock(nn.Module):
    def __init__(self, layer_id: int, args: ModelArgs):
        super().__init__()

        self.n_heads = args.n_heads
        self.dim = args.dim
        # 4096 // 32
        self.head_dim = args.dim // args.n_heads
        self.attention = Attention(args)



class Transformer(nn.Module):
    def __init__(self, params: ModelArgs):
        super().__init__()

        self.n_heads = params.n_heads
        self.vocab_size = params.vocab_size
        self.n_layers = params.n_layers

        self.tok_embeddings = VocabParallelEmbedding(params.vocab_size,
                                                     params.dim, 
                                                     init_method=lambda x: x)
        self.layers = torch.nn.ModuleList()

        for layer_id in range(params.n_layers):
            self.layers.append(TransformerBlock(layer_id, params))



class LLama(object):

    def __init__(self, model: Transformer):
        pass
